//////////////////////////////////////////////////////////////////////////
//
// This file is a part of XORT project.
//
// PROJECT:         Xort compiler
// FILE:            lex.cpp
// AUTHOR:          Vladimir Gumenuk
// DATE:            22-Mar-2008
// MODIFICATIONS:
//
//
//     Copyright(c) 2014, Vladimir Gumenuk
//     All rights reserved.
//
//     Redistribution and use in source and binary forms, with or without modification, 
//     are permitted provided that the following conditions are met :
//
//     1. Redistributions of source code must retain the above copyright notice, 
//        this list of conditions and the following disclaimer.
//     2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and 
//        the following disclaimer in the documentation and / or other materials provided with the distribution.
//     3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse 
//        or promote products derived from this software without specific prior written permission.
//
//     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
//     ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 
//     OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 
//     THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 
//     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 
//     OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
//     HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
//     OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 
//     EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
//////////////////////////////////////////////////////////////////////////

#include "xort.h"

STATIC const dword  LINE_START      = 0;
STATIC const dword  POS_START       = 0;
STATIC BYTE_ARRAY   g_csBuffer;    // program text
STATIC DWORD_ARRAY  g_arLines;     // contains the position of each line in the text buffer
STATIC Token        g_Tok;         // current token;
STATIC Token        g_PTok;        // previous token;
STATIC Token        g_PPTok;       // prev prev token;
STATIC dword        g_dwBufPtr      = 0; // pointer in text buffer
STATIC dword        g_dwLine        = LINE_START; // current line No
STATIC dword        g_dwPos         = POS_START; // current position No
STATIC dword        g_dwPrevLine    = LINE_START; // previous line No
STATIC dword        g_dwPrevPos     = POS_START; // previous position No
STATIC dword        g_dwPrevBufPtr  = 0; // previous pointer in text buffer
STATIC string       g_csFileName;        // name of the input file

int lex_Input( string csFile )
{
    File fIn;
    if( !fIn.Open( csFile, "rb" ) )
        return 0;

    dword dwRead, i;

    g_arLines.add( 0 ); // line 0, starts from 0 :)

    dword dwFileSize = fIn.Length();
#ifndef CPP_COMPILER
    DbgPrint( "lex_Input() -> ENTER: %@, size: %@ bytes", csFile, dwFileSize );
#endif
    dwRead = fIn.Read( g_csBuffer, dwFileSize ); // try to read entire file at once
    if( dwRead )
    {
        i = 0;
        dword dwCount = g_csBuffer.count();
        while( i < dwCount )
        {
            byte ch = g_csBuffer.get( i );
            if( ch == '\n' )
            {
                g_arLines.add( i + 1 ); // save line position
            }
            ++i;
        }
    } else {
#ifdef CPP_COMPILER
        message( "Can not read the file: '%s'", csFile.GetBuffer() );
#else
        message( "Can not read the file: '%@'", csFile );
#endif
        return ERR_CANT_READ_FILE;
    }
    g_csBuffer.add( 0 ); // add final 0

    g_csFileName.Assign( fIn.Name() );
    
    fIn.Close();
    DbgPrint( "lex_Input() -> LEAVE, lines: %@", g_arLines.count() );
    return XORT_OK;
}

string lex_getFileName()
{
    return g_csFileName;
}

STATIC void blanks()
{
    //DbgPrint( "blanks() -> ENTER - Line: 0x%@, pos: 0x%@, bufIdx: 0x%@", g_dwLine, g_dwPos, g_dwBufPtr );
    
    byte ch = g_csBuffer.get( g_dwBufPtr );
    while( ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t' )
    {
        if( ch == '\n' )
        {
            ++g_dwLine;
            g_dwPos = POS_START;
        }
        else
            ++g_dwPos;

        ++g_dwBufPtr;
 
        if( g_csBuffer.get( g_dwBufPtr ) == 0 ) {
            break;
        }

        ch = g_csBuffer.get( g_dwBufPtr );
    }

    //DbgPrint( "blanks() -> LEAVE - Line: 0x%@, pos: 0x%@, bufIdx: 0x%@", g_dwLine, g_dwPos, g_dwBufPtr );
}

STATIC byte CH() // returns char at current position
{
    return g_csBuffer.get( g_dwBufPtr );
}

STATIC byte NCH() // returns char at 1 position ahead
{
    return g_csBuffer.get( g_dwBufPtr+1 );
}

STATIC byte GCH() // advances internal pointer index and returns char at new position
{
    ++g_dwBufPtr;
    ++g_dwPos;
    return g_csBuffer.get( g_dwBufPtr );
}

STATIC void SavePrevLinePos()
{
    g_dwPrevLine   = g_dwLine;
    g_dwPrevPos    = g_dwPos;
    g_dwPrevBufPtr = g_dwBufPtr;
}

STATIC void SetCurLinePosToCurToken()
{
    g_Tok.m_dwLine = g_dwLine;
    g_Tok.m_dwPos  = g_dwPos;
}

STATIC void SkipComment()
{
    //DbgPrint( "SkipCmment() -> ENTER - Line: 0x%@, pos: 0x%@, bufIdx: 0x%@", g_dwLine, g_dwPos, g_dwBufPtr );
    byte ch;
    if( CH() == '/' && NCH() == '/' ) {
        ch = GCH();
        while( ch != '\n' )
        {
            ch = GCH();
            if( ch == '\n' ) {
                ++g_dwLine;
                g_dwPos = POS_START;
            }

            if( ch == 0 )
                break;
        }
        GCH();
    } else if( CH() == '/' && NCH() == '*' ) {
        GCH();
        do
        {
            ch = GCH();
            if( ch == '\n' ) {
                ++g_dwLine;
                g_dwPos = POS_START;
            }

            if( ch == 0 ) {
                break;
            }
        } while( !( ch == '*' && NCH() == '/' ) );

        if( ch != 0 ) {
            GCH(); GCH();
        }
    }
    //DbgPrint( "SkipCmment() -> LEAVE - Line: 0x%@, pos: 0x%@, bufIdx: 0x%@", g_dwLine, g_dwPos, g_dwBufPtr );
}
// method returns 1 if current token is the object member, 0 otherwise
int lex_ConfirmMember( SYMBOLLIST symList, SymbolRef Var, SymbolRef ObjBase )
{
    if( g_Tok.m_TT == tt_ident && g_PTok.m_TT == tt_delim && g_PTok.m_dwTokenCode == tk_ptr ) { //case obj.CurrentTok
        // in this case prev token must be object!
        SymbolRef pObject = FindSymbol( symList, g_PPTok.m_csVal, FALSE ); // must be global
        if( !pObject.isNULL() ) {
            SymbolRef pType = FindType( symList, pObject.m_dwType );
            if( !pType.isNULL() ) {
                SymbolRef pMember = FindSymbol( pType.m_listMembers, g_Tok.m_csVal, TRUE );
                if( !pMember.isNULL() ) {
                    pMember.Copy( Var );
                    return 1;
                } else { // if one of these conditions is false - means error in compiler!
                    return ERR_INTERNAL;
                }
            } else {
                return ERR_INTERNAL;
            }
        } else { // this means, that we are parsing something like obj1.obj2.obj3...
                 // and pObjBase must point to the prev object type;
            if( !ObjBase.isNULL() ) { // if NULL - called from lex, otherwise - from expr_primary
                SymbolRef pType = FindType( symList, ObjBase.m_dwType );
                if( !pType.isNULL() ) {
                    SymbolRef pMember = FindSymbol( pType.m_listMembers, g_Tok.m_csVal, TRUE );
                    if( !pMember.isNULL() ) {
                        pMember.Copy( Var );
                        return 1;
                    } else { // if one of these conditions is false - means error in compiler!
                        return ERR_INTERNAL;
                    }
                } else {
                    return ERR_INTERNAL;
                }
            } else {
                return ERR_INTERNAL;
            }
        }
    }
    return 0;
}

// This function will be as test for optimizer
STATIC void DumpCurrentToken()
{
    //CheckStringType();
    //g_Tok.Dump();
}

Token lex_GetToken( SYMBOLLIST symList )
{
    //DbgPrint( "lex_GetToken() -> ENTER" );

    byte ch;
    g_PTok.Copy( g_PPTok ); // save 2 last tokens!
    g_Tok.Copy( g_PTok );
    g_Tok.Clear();
    SavePrevLinePos();
    
    if( CH() == 0 ) // eof?
    {
        DbgPrint( "lex_GetToken() -> EOF! 1" );
        g_Tok.m_TT = tt_eof; DumpCurrentToken();
        return g_Tok;
    }

    do 
    {
        blanks();
        SkipComment();
        blanks();
    } while( (CH() == '/' && NCH() == '/') || (CH() == '/' && NCH() == '*') );

    if( CH() == 0 ) // eof?
    {
        DbgPrint( "lex_GetToken() -> EOF! 2" );
        g_Tok.m_TT = tt_eof; DumpCurrentToken();
        return g_Tok;
    }

    ch = CH();
//-----------------------------------------------------------------------------
    if( x_isdelim( ch ) )
    { 
        if( ch == '!' || ch == '>' || ch == '<' || ch == '=' || ch == '|' || ch == '&' || 
            ch == '+' || ch == '-' || ch == '/' || ch == '%' || ch == '*' || ch == '^' )
        {
            GCH();
            byte nch = CH();
            if( nch == '=' )
            {
                if( ch == '!' ) {
                    g_Tok.m_dwTokenCode = tk_notequal;
                } else if( ch == '>' ) {
                    g_Tok.m_dwTokenCode = tk_greaterequal; 
                } else if( ch == '<' ) {
                    g_Tok.m_dwTokenCode = tk_lessequal;
                } else if( ch == '=' ) {
                    g_Tok.m_dwTokenCode = tk_equal;
                } else if( ch == '|' ) {
                    g_Tok.m_dwTokenCode = tk_assignbinor;
                } else if( ch == '&' ) {
                    g_Tok.m_dwTokenCode = tk_assignbinand;
                } else if( ch == '+' ) {
                    g_Tok.m_dwTokenCode = tk_assignplus;
                } else if( ch == '-' ) {
                    g_Tok.m_dwTokenCode = tk_assignminus;
                } else if( ch == '*' ) {
                    g_Tok.m_dwTokenCode = tk_assignmul;
                } else if( ch == '/' ) {
                    g_Tok.m_dwTokenCode = tk_assigndiv;
                } else if( ch == '%' ) {
                    g_Tok.m_dwTokenCode = tk_assignmod;
                } else if( ch == '^' ) {
                    g_Tok.m_dwTokenCode = tk_assignxor;
                }
                if( g_Tok.m_dwTokenCode != 0 ) {
                    GCH();
                }
            } else if( nch == '|' && ch == '|' ) {
                g_Tok.m_dwTokenCode = tk_logor; GCH();
            } else if( nch == '&' && ch == '&' ) {
                g_Tok.m_dwTokenCode = tk_logand; GCH();
            } else if( nch == '>' && ch == '>' ) {
                g_Tok.m_dwTokenCode = tk_shiftright; GCH();
            } else if( nch == '<' && ch == '<' ) {
                g_Tok.m_dwTokenCode = tk_shiftleft; GCH();
            } else if( nch == '+' && ch == '+' ) {
                g_Tok.m_dwTokenCode = tk_inc; GCH();
            } else if( nch == '-' && ch == '-' ) {
                g_Tok.m_dwTokenCode = tk_dec; GCH();
            } else {
                if( ch == '!' ) {
                    g_Tok.m_dwTokenCode = tk_not;
                } else if( ch == '>' ) {
                    g_Tok.m_dwTokenCode = tk_greater;
                } else if( ch == '<' ) {
                    g_Tok.m_dwTokenCode = tk_less;
                } else if( ch == '=' ) {
                    g_Tok.m_dwTokenCode = tk_assign;
                } else if( ch == '|' ) {
                    g_Tok.m_dwTokenCode = tk_binor;
                } else if( ch == '&' ) {
                    g_Tok.m_dwTokenCode = tk_binand;
                } else if( ch == '+' ) {
                    g_Tok.m_dwTokenCode = tk_plus;
                } else if( ch == '-' ) {
                    g_Tok.m_dwTokenCode = tk_minus;
                } else if( ch == '*' ) {
                    g_Tok.m_dwTokenCode = tk_mul;
                } else if( ch == '/' ) {
                    g_Tok.m_dwTokenCode = tk_div;
                } else if( ch == '%' ) {
                    g_Tok.m_dwTokenCode = tk_mod;
                } else if( ch == '^' ) {
                    g_Tok.m_dwTokenCode = tk_xor;
                }
            }

            if( g_Tok.m_dwTokenCode == tk_shiftleft || g_Tok.m_dwTokenCode == tk_shiftright )
            {
                nch = CH();
                if( nch == '=' )
                {
                    GCH();
                    if( g_Tok.m_dwTokenCode == tk_shiftleft ) {
                        g_Tok.m_dwTokenCode = tk_assignshl;
                    } else {
                        g_Tok.m_dwTokenCode = tk_assignshr;
                    }
                }
            }
        } else {
            if( ch == ';' ) {
                g_Tok.m_dwTokenCode = tk_semicolon;
            } else if( ch == '(' ) {
                g_Tok.m_dwTokenCode = tk_parenthopen;
            } else if( ch == ')' ) {
                g_Tok.m_dwTokenCode = tk_parenthclose;
            } else if( ch == ',' ) {
                g_Tok.m_dwTokenCode = tk_coma;
            } else if( ch == '[' ) {
                g_Tok.m_dwTokenCode = tk_sqrbropen;
            } else if( ch == ']' ) {
                g_Tok.m_dwTokenCode = tk_sqrbrclose;
            } else if( ch == '{' ) {
                g_Tok.m_dwTokenCode = tk_blkopen;
            } else if( ch == '}' ) {
                g_Tok.m_dwTokenCode = tk_blkclose;
            } else if( ch == '?' ) {
                g_Tok.m_dwTokenCode = tk_quest;
            } else if( ch == ':' ) {
                g_Tok.m_dwTokenCode = tk_colon;
            } else if( ch == '~' ) {
                g_Tok.m_dwTokenCode = tk_tilda;
            } else if( ch == '.' ) {
                g_Tok.m_dwTokenCode = tk_ptr;
            }
            if( g_Tok.m_dwTokenCode != 0 ) {
                GCH();

                if( g_Tok.m_dwTokenCode == tk_ptr ) {
                    if( CH() == '.' && NCH() == '.' ) {
                        g_Tok.m_dwTokenCode = tk_elipsis;
                        GCH(); GCH();
                    }
                }
            }
        }

        if( g_Tok.m_dwTokenCode == 0 ) {
            x_error( ERR_UNKNOWN_TOKEN, g_dwLine, g_dwPos );
            g_Tok.m_TT = tt_unknown; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        }

        g_Tok.m_TT = tt_delim; SetCurLinePosToCurToken(); DumpCurrentToken();
        return g_Tok;
    }
//-----------------------------------------------------------------------------
    if( ch == '\'' ) // character
    {
        GCH();
        ch = CH(); // get char
        SetCurLinePosToCurToken();

        byte bCode;
        if( ch == '\\' ) // test for escaped character:
        {
            ch = GCH();
            switch( ch )
            {
            case 'a':  bCode = 0x07; break;
            case 'b':  bCode = 0x08; break;
            case 'f':  bCode = 0x0C; break;
            case 'n':  bCode = 0x0A; break;
            case 'r':  bCode = 0x0D; break;
            case 't':  bCode = 0x09; break;
            case 'v':  bCode = 0x0B; break;
            case '\'': bCode = 0x27; break;
            case '\"': bCode = 0x22; break;
            case '\\': bCode = 0x5C; break;
            case '\?': bCode = 0x3F; break;
            case '0':  bCode = 0x00; break;
            default:
                // some compilers handles undefined character as character itself,
                // however it is not required, but anyway, initialize code by char value
                bCode = ch;
                break;
            }
        } else {
            bCode = ch;
        }

        if( NCH() != '\'' ) {
            x_error( ERR_MISSING_QUOTE, g_dwLine, g_dwPos );
            g_Tok.m_TT = tt_unknown; g_Tok.m_dwTokenCode = tk_unknown;
        } else {
            SetCurLinePosToCurToken();
            g_Tok.m_TT = tt_digit; g_Tok.m_dwTokenCode = tk_number;
            g_Tok.m_Val.bType = t_u1; g_Tok.m_Val._u1 = bCode;
            GCH(); GCH();
        }
        DumpCurrentToken();
        return g_Tok;
    }
//-----------------------------------------------------------------------------
    if( ch == '"' ) // "string"
    { 
        ch = GCH();
        while( ch != '"' && ch != '\r' )
        {
            if( ch == '\\' ) // handle escape sequence
            {
                byte bCode = 0xFF;
                byte nch = NCH();
                switch( nch )
                {
                case 'a':  bCode = 0x07; break;
                case 'b':  bCode = 0x08; break;
                case 'f':  bCode = 0x0C; break;
                case 'n':  bCode = 0x0A; break;
                case 'r':  bCode = 0x0D; break;
                case 't':  bCode = 0x09; break;
                case 'v':  bCode = 0x0B; break;
                case '\'': bCode = 0x27; break;
                case '\"': bCode = 0x22; break;
                case '\\': bCode = 0x5C; break;
                case '\?': bCode = 0x3F; break;
                case '0':  bCode = 0x00; break;
                }
                if( bCode != 0xFF ) {
                    g_Tok.m_csVal.Append( bCode );
                    GCH();
                } else {
                    g_Tok.m_csVal.Append( ch );
                }
                ch = GCH();
            } else {
                g_Tok.m_csVal.Append( ch );
                ch = GCH();
            }
        }
        if( ch == '\r' ) {
            x_error( ERR_STRING_SYNTAX, g_dwLine, g_dwPos );
            g_Tok.m_TT = tt_unknown; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        }
        GCH();
        g_Tok.m_dwLiteralIdx = lit_Add( g_Tok.m_csVal );
        g_Tok.m_TT = tt_literal; SetCurLinePosToCurToken(); DumpCurrentToken();
        return g_Tok;
    }
//-----------------------------------------------------------------------------
    if( x_isdigit( ch ) )
    { 
        int isHex = FALSE;

        g_Tok.m_TT = tt_digit;
        g_Tok.m_dwTokenCode = tk_number;

        if( ch == '0' && ( NCH() == 'x' || NCH() == 'X' ) ) // hex? 0x 0X ?
        {
            GCH(); // eat 0x
            ch = GCH();
            if( !x_ishexdigit( ch ) && !x_isdigit( ch ) ) {
                x_error( ERR_HEXDIGIT_SYNTAX, g_dwLine, g_dwPos );
                g_Tok.m_TT = tt_unknown; SetCurLinePosToCurToken(); DumpCurrentToken();
                return g_Tok;
            }

            isHex = TRUE;

            qword qwValue = 0;
            while( x_ishexdigit( ch ) || x_isdigit( ch ) ) // collect all hex digits
            {
                g_Tok.m_csVal.Append( ch );
                dword dwDigit = x_getHexValue( ch );
                qwValue = qwValue * 16 + dwDigit;

                if( !x_ishexdigit( NCH() ) && !x_isdigit( NCH() ) ) {
                    break;
                }

                ch = GCH();
            } 
            
            g_Tok.m_Val.bType = t_u8;
            g_Tok.m_Val._u8 = qwValue;

        } else { // decimal digits

            while( x_isdigit( ch ) ) // collect all decimal digits
            {
                g_Tok.m_csVal.Append( ch );
                if( !x_isdigit( NCH() ) ) {
                    break;
                }
                ch = GCH();
            }
        }

        ch = NCH();

        if( ch == '.' )
        {
            if( isHex ) { // hex number can not be with floating point
                x_error( ERR_HEXDIGIT_SYNTAX, g_dwLine, g_dwPos );
                g_Tok.m_TT = tt_unknown; SetCurLinePosToCurToken(); DumpCurrentToken();
                return g_Tok;
            }

            g_Tok.m_csVal.Append( ch );
            GCH();
            ch = GCH();
            if( !x_isdigit( ch ) ) {
                x_error( ERR_FLOATINGP_SYNTAX, g_dwLine, g_dwPos );
                g_Tok.m_TT = tt_unknown; SetCurLinePosToCurToken(); DumpCurrentToken();
                return g_Tok;
            }

            while( x_isdigit( ch ) ) // collect exponent digits
            {
                g_Tok.m_csVal.Append( ch );
                if( !x_isdigit( NCH() ) ) {
                    break;
                }
                ch = GCH();
            }

            ch = NCH();

            if( ch == 'F' ) {
                g_Tok.m_Val.bType = t_r4;
                g_Tok.m_Val._r4 = g_Tok.m_csVal.floatValue();
                GCH();
            } else if( ch == 'R' ) {
                g_Tok.m_Val.bType = t_r8;
                g_Tok.m_Val._r8 = g_Tok.m_csVal.doubleValue();
                GCH();
            } else { // double by default
                g_Tok.m_Val.bType = t_r8;
                g_Tok.m_Val._r8 = g_Tok.m_csVal.doubleValue();
            }

            GCH();
            g_Tok.m_dwTokenCode = tk_realnumber; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( ch == 'B' ) { // byte
            g_Tok.m_Val.bType = t_u1;
            g_Tok.m_Val._u1 = g_Tok.m_csVal.intValue();
            GCH();
        } else if( ch == 'C' ) { // char
            g_Tok.m_Val.bType = t_s1;
            g_Tok.m_Val._s1 = g_Tok.m_csVal.intValue();
            GCH();
        } else if( ch == 'S' ) { // short
            g_Tok.m_Val.bType = t_s2;
            g_Tok.m_Val._s2 = g_Tok.m_csVal.intValue();
            GCH();
        } else if( ch == 'W' ) { // word
            g_Tok.m_Val.bType = t_u2;
            g_Tok.m_Val._u2 = g_Tok.m_csVal.intValue();
            GCH();
        } else if( ch == 'I' ) { // int
            g_Tok.m_Val.bType = t_s4;
            g_Tok.m_Val._s4 = g_Tok.m_csVal.intValue();
            GCH();
        } else if( ch == 'U' ) { // dword
            g_Tok.m_Val.bType = t_u4;
            g_Tok.m_Val._u4 = g_Tok.m_csVal.intValue();
            GCH();
        } else if( ch == 'L' ) { // long
            g_Tok.m_Val.bType = t_s8;
            g_Tok.m_Val._s8 = g_Tok.m_csVal.longValue();
            GCH();
        } else if( ch == 'Q' ) { // qword
            g_Tok.m_Val.bType = t_u8;
            g_Tok.m_Val._u8 = g_Tok.m_csVal.longValue();
            GCH();
        } else { // int by default
            if( !isHex ) { // hex handled above
                g_Tok.m_Val.bType = t_s4;
                g_Tok.m_Val._s4 = g_Tok.m_csVal.intValue();
            }
        }

        GCH();
        SetCurLinePosToCurToken(); DumpCurrentToken();
        return g_Tok;
    }
//-----------------------------------------------------------------------------
    if( x_isalpha( ch ) ) // variable or command
    { 
        while( x_isalnum( ch ) )
        {
            g_Tok.m_csVal.Append( ch );
            ch = GCH();
        }

        // decide what is it - command, var or typename?
        if( g_Tok.m_csVal.Compare( "if" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_if; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "else" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_else; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "while" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_while; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "do" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_do; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "break" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_break; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "struct" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_object; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "continue" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_continue; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "typename" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_typename; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "return" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_return; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "for" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_for; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "const" ) == 0 ) {
            g_Tok.m_TT = tt_modifier; g_Tok.m_dwTokenCode = tk_const; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "typedef" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_typedef; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "array" ) == 0 ) {
            g_Tok.m_TT = tt_template; g_Tok.m_dwTokenCode = tk_array; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "list" ) == 0 ) {
            g_Tok.m_TT = tt_template; g_Tok.m_dwTokenCode = tk_list; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "switch" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_switch; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "case" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_case; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "default" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_default; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "enum" ) == 0 ) {
            g_Tok.m_TT = tt_stmt; g_Tok.m_dwTokenCode = tk_enum; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "NULL" ) == 0 ) {
            g_Tok.m_TT = tt_null; g_Tok.m_dwTokenCode = tk_nullptr; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "import" ) == 0 ) {
            g_Tok.m_TT = tt_attr; g_Tok.m_dwTokenCode = tk_a_import; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "name" ) == 0 ) {
            g_Tok.m_TT = tt_attr; g_Tok.m_dwTokenCode = tk_a_name; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "module" ) == 0 ) {
            g_Tok.m_TT = tt_attr; g_Tok.m_dwTokenCode = tk_a_module; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "callconv" ) == 0 ) {
            g_Tok.m_TT = tt_attr; g_Tok.m_dwTokenCode = tk_a_callconv; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "platform" ) == 0 ) {
            g_Tok.m_TT = tt_attr; g_Tok.m_dwTokenCode = tk_a_platform; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        } else if( g_Tok.m_csVal.Compare( "export" ) == 0 ) {
            g_Tok.m_TT = tt_attr; g_Tok.m_dwTokenCode = tk_a_export; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        }

        SymbolRef pSym = FindSymbol( symList, g_Tok.m_csVal, FALSE );
        if( !pSym.isNULL() )
        {
            if( pSym.m_SymKind == sk_typename || pSym.m_SymKind == sk_obj ) {
                g_Tok.m_TT = tt_typename;
            } else {
                g_Tok.m_TT = tt_ident;
            }

            g_Tok.m_dwTokenCode = pSym.m_dwType; SetCurLinePosToCurToken();

            Symbol sym;
            if( lex_ConfirmMember( symList, sym, getNULLSymbol() ) == 1 ) { //case obj.CurrentTok
                g_Tok.m_dwTokenCode = sym.m_dwType; // set the actual type of the member
            }
            DumpCurrentToken();
            return g_Tok;
        } else {
            g_Tok.m_TT = tt_ident; g_Tok.m_dwTokenCode = tk_unknown; SetCurLinePosToCurToken(); DumpCurrentToken();
            return g_Tok;
        }
    }

    x_error( ERR_UNKNOWN_TOKEN, g_dwLine, g_dwPos );
    g_Tok.m_TT = tt_unknown; SetCurLinePosToCurToken(); DumpCurrentToken();
    return g_Tok;
}

Token lex_GetLastToken()
{
    return g_Tok;
}

void lex_PutBack() // should be used outside this module!
{
    g_dwLine   = g_dwPrevLine;
    g_dwPos    = g_dwPrevPos;
    g_dwBufPtr = g_dwPrevBufPtr;
}

int lex_match( SYMBOLLIST symList, dword tokType, dword dwTokCode, int bTestLastTok )
{
    if( bTestLastTok ) {
        Token Tok = lex_GetLastToken();
        if( Tok.m_TT != tokType || Tok.m_dwTokenCode != dwTokCode ) { 
            x_error( ERR_UNEXPECTED_TOKEN, Tok.m_dwLine, Tok.m_dwPos );
            return ERR_SYNTAX;
        }
    } else { // I know, this is duplication of verification but, can't improve this right now: compilation error - reference must be initialized!!
        Token Tok = lex_GetToken( symList );
        if( Tok.m_TT != tokType || Tok.m_dwTokenCode != dwTokCode ) {
            x_error( ERR_UNEXPECTED_TOKEN, Tok.m_dwLine, Tok.m_dwPos );
            return ERR_SYNTAX;
        }
    }

    return XORT_OK;
}

string lex_GetLine( dword dwIdx )
{
    string csRet;

    if( dwIdx >= g_arLines.count() )
        return csRet;

    dword i = g_arLines.get( dwIdx );
    byte ch = g_csBuffer.get( i );

    while( ch != '\r' )
    {
        csRet.Append( ch );
        ++i;
        ch = g_csBuffer.get( i );
        if( ch == 0 )
            break;
    }

    return csRet;
}